import codecs
import os


def read_url_list(url_list_file_path, dedupe=True):
    if os.path.isfile(url_list_file_path):
        with codecs.open(url_list_file_path, mode='r', encoding='utf-8') as f:
            url_list = f.readlines()
        url_list = list(filter(lambda x: not x.startswith("#"), url_list))  # 先fitler掉#开头的行注释
        url_list = [item.split(" -> ")[0].strip() for item in url_list]  # 这个是保留单纯url的url_list
    else:
        url_list = []
    if dedupe:
        url_list = list(set(url_list))
        print(f"dedupe url list from: {url_list_file_path}, {len(url_list)} -> {len(url_list)} ")  # dedupe
    return url_list


def append_url(url_list_file_path, url, mode="a+"):
    with codecs.open(url_list_file_path, mode=mode, encoding='utf-8') as f:  # 完成一个写一个
        f.write(url + "\n")


def append_url_with_subsidiary(url_list_file_path, url, subsidiary, mode="a+"):
    with codecs.open(url_list_file_path, mode=mode, encoding='utf-8') as f:  # 完成一个写一个
        f.write(url + " -> " + subsidiary + "\n")


def append_url_list(url_list_file_path, url_list):
    write_url_list(url_list_file_path, url_list, mode="a", dedupe=False)


def write_url_list(url_list_file_path, url_list, mode="w", dedupe=True):
    if dedupe:
        url_list = list(set(url_list))
    with codecs.open(url_list_file_path, mode=mode, encoding='utf-8') as f:
        f.writelines([url + "\n" for url in url_list])


def append_url_list_with_subsidiary(url_list_file_path, url_subsidiary_list, dedupe=True):
    write_url_list_with_subsidiary(url_list_file_path, url_subsidiary_list, mode="a", dedupe=dedupe)


def write_url_list_with_subsidiary(url_list_file_path, url_subsidiary_list, mode="w", dedupe=True):
    if dedupe:
        url_subsidiary_list = list(set(url_subsidiary_list))
    with codecs.open(url_list_file_path, mode=mode, encoding='utf-8') as f:
        f.writelines([" -> ".join(inner_list) + "\n" for inner_list in url_subsidiary_list])